package eu.best.sercheng.fetcher;

import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.URL;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import edu.princeton.cogsci.wn.WordNetImpl;
import eu.best.sercheng.frontier.Frontiere;
import eu.best.sercheng.model.dao.MainDao;

public class Fetcher extends Thread {

	public URL urlLink = null;
	public boolean start;
	public Map<String, Date> cache;
	public Frontiere frontier;
	DateFormat dateFormat;
	Calendar cal;
	MainDao dao;

	public Fetcher(URL link, Frontiere fr, MainDao md) {
		start = true;
		urlLink = link;
		cache = new HashMap<String, Date>();
		frontier = fr;
		dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
		dao = md;
	}

	public void run() {
		String absoluteUrl = null;
		List<URL> urlList = new LinkedList<URL>();

		if (urlLink == null) {
			urlLink = frontier.getLink();
		}
		while (true) {
			try {
				if (urlLink != null) {
					
					// getting new link to parse from URL Frontier
					if(!start) urlLink = frontier.getLink();
					else start = false;
					
					// using proxy
					URL url = new URL(urlLink.toExternalForm());
					Proxy proxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress("proxy.cs.tuiasi.ro", 8080));
					HttpURLConnection uc = (HttpURLConnection) url.openConnection(proxy);

					uc.connect();
					if (!uc.getContentType().contains("text/html")) continue;
					String line = null;
					StringBuffer tmp = new StringBuffer();
					BufferedReader in = new BufferedReader(new InputStreamReader(uc.getInputStream()));
					while ((line = in.readLine()) != null) {
						tmp.append(line);
					}
					// end of proxy using

					Document doc = Jsoup.parse(String.valueOf(tmp),url.toExternalForm());

					Elements links = doc.select("a");

					for (Element link : links) {
						absoluteUrl = link.attr("abs:href");
						if (absoluteUrl.length() != 0)
							urlList.add(new URL(absoluteUrl));
					}

					// sending all new links to URL frontier
					if(urlList.size()!=0)frontier.pushLink(urlList);
					urlList.clear();

					
					//getting content from body
					doc = Jsoup.parseBodyFragment(String.valueOf(tmp));
					Element body = doc.body();
					
					List<String> listOfContent = new LinkedList<String>();
					String[] tableOfContents = body.text().split(" |;|\\.|,|/|\\)|\\(|:|'|\"|-|_|\\+|=|!|\\?|”|„|”");
					for(String i : tableOfContents){
						i = WordNetImpl.toCanonicalForm(i);
						if(!i.equals(" ")) listOfContent.add(i);
					}
			
						//writing into db
						System.out.println(urlLink.toExternalForm() + " " + listOfContent);
						dao.putWordsToUrl(urlLink.toExternalForm(), listOfContent);		

					uc.disconnect();
				} else {
					sleep(1000);
					urlLink = frontier.getLink();
				}
			} catch (FileNotFoundException e) {
				System.err.println("Site doesn't exist " + urlLink);
				urlLink = null;
			} catch (IOException e) {
				System.err.println("Service Unavailable " + urlLink);
				urlLink = null;
			} catch (Exception e) {
				//e.printStackTrace();
				System.err.println("sth " + urlLink);
				urlLink = null;
			}
		}

	}
	
	public Map<String,String> robotsHandle(String url){
		Map<String,String> allowDisallow = null;
		try{
			allowDisallow = new HashMap<String,String>();
			
			//reading robots.txt file from the web
			BufferedReader br = new BufferedReader(new FileReader(url + "robots.txt"));
			String str = "";
			
			//putting Allow and Disallow into map
			while ((str = br.readLine()) != null) {
				//System.out.println(str);
				if(str.contains("Disallow")) allowDisallow.put(str.split(": ")[1],"Disallow");
				else if(str.contains("Disallow")) allowDisallow.put(str.split(": ")[1],"Allow");
			}
			
			br.close();
			
		}catch(Exception e){
			System.err.println("RobotException");
		}
		return allowDisallow;
	}

}
